From: tsteven4 Date: Fri, 5 Apr 2019 14:43:38 +0000 (-0600) Subject: split csv lines as QStrings and return a list of all fields. (#332) X-Git-Tag: archive/raspbian/1.10.0+ds-2+rpi1~1^2~12^2~8^2~12 X-Git-Url: https://dgit.raspbian.org/%22http:/www.example.com/cgi/%22https://%22Program/%22http:/www.example.com/cgi/%22https:/%22Program?a=commitdiff_plain;h=f890a1fb364faae6a1a683562178138cea625613;p=gpsbabel.git split csv lines as QStrings and return a list of all fields. (#332) * introdcue csv_linesplit Which is like csv_lineparse, except it gives you a list of all the values, and it processing the line as a QString. * introduce gpsbabel::TextStream, use it in unicsv reader, ozi. * convert unicsv writer to textstream. * if a codec is not found list available. * switch unicsv to CET_CHARSET_UTF8 to avoid undesired fs conversions. All conversions are handled by the codec used by qtextstream. * output boms with non utf8 unicode codecs. * use rfc4180 dequote method with unicsv. * add test for csv quoting w RFC4180. --- diff --git a/CMakeLists.txt b/CMakeLists.txt index cbea4eb58..0a2f1bdad 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -96,6 +96,7 @@ set(SUPPORT formspec.cc xmltag.cc cet.cc cet_util.cc fatal.cc rgbcolors.cc inifile.cc garmin_fs.cc units.cc gbser.cc gbfile.cc parse.cc session.cc main.cc globals.cc + src/core/textstream.cc src/core/usasciicodec.cc src/core/xmlstreamwriter.cc ) @@ -164,6 +165,7 @@ set(HEADERS src/core/datetime.h src/core/file.h src/core/logging.h + src/core/textstream.h src/core/usasciicodec.h src/core/xmlstreamwriter.h src/core/xmltag.h diff --git a/GPSBabel.pro b/GPSBabel.pro index 40fedbebb..87c63c32c 100644 --- a/GPSBabel.pro +++ b/GPSBabel.pro @@ -85,6 +85,7 @@ SUPPORT = route.cc waypt.cc filter_vecs.cc util.cc vecs.cc mkshort.cc \ formspec.cc xmltag.cc cet.cc cet_util.cc fatal.cc rgbcolors.cc \ inifile.cc garmin_fs.cc units.cc gbser.cc \ gbfile.cc parse.cc session.cc main.cc globals.cc \ + src/core/textstream.cc \ src/core/usasciicodec.cc \ src/core/xmlstreamwriter.cc @@ -136,7 +137,7 @@ HEADERS = \ session.h \ shapelib/shapefil.h \ strptime.h \ - xcsv.h \ + xcsv.h \ xmlgeneric.h \ zlib/crc32.h \ zlib/deflate.h \ @@ -152,6 +153,7 @@ HEADERS = \ src/core/datetime.h \ src/core/file.h \ src/core/logging.h \ + src/core/textstream.h \ src/core/usasciicodec.h \ src/core/xmlstreamwriter.h \ src/core/xmltag.h diff --git a/Makefile.in b/Makefile.in index 1b13a2af4..46202af61 100644 --- a/Makefile.in +++ b/Makefile.in @@ -121,9 +121,9 @@ LIBOBJS = route.o waypt.o filter_vecs.o util.o vecs.o mkshort.o \ formspec.o xmltag.o cet.o cet_util.o fatal.o rgbcolors.o \ inifile.o garmin_fs.o units.o @GBSER@ gbser.o \ gbfile.o parse.o session.o \ + src/core/textstream.o \ + src/core/usasciicodec.o \ src/core/xmlstreamwriter.o \ - src/core/usasciicodec.o\ - src/core/ziparchive.o \ $(GARMIN) $(JEEPS) $(SHAPE) @ZLIB@ @MINIZIP@ $(FMTS) $(FILTERS) OBJS = main.o globals.o $(LIBOBJS) @FILEINFO@ @@ -459,7 +459,7 @@ cst.o: cst.cc defs.h config.h zlib/zlib.h zlib/zconf.h cet.h inifile.h \ gbfile.h session.h src/core/datetime.h src/core/optional.h cet_util.h csv_util.o: csv_util.cc defs.h config.h zlib/zlib.h zlib/zconf.h cet.h \ inifile.h gbfile.h session.h src/core/datetime.h src/core/optional.h \ - csv_util.h + csv_util.h src/core/logging.h delgpl.o: delgpl.cc defs.h config.h zlib/zlib.h zlib/zconf.h cet.h \ inifile.h gbfile.h session.h src/core/datetime.h src/core/optional.h destinator.o: destinator.cc defs.h config.h zlib/zlib.h zlib/zconf.h \ @@ -846,7 +846,7 @@ osm.o: osm.cc defs.h config.h zlib/zlib.h zlib/zconf.h cet.h inifile.h \ xmlgeneric.h ozi.o: ozi.cc defs.h config.h zlib/zlib.h zlib/zconf.h cet.h inifile.h \ gbfile.h session.h src/core/datetime.h src/core/optional.h csv_util.h \ - jeeps/gpsmath.h jeeps/gpsport.h src/core/file.h + jeeps/gpsmath.h jeeps/gpsport.h src/core/textstream.h src/core/file.h parse.o: parse.cc defs.h config.h zlib/zlib.h zlib/zconf.h cet.h \ inifile.h gbfile.h session.h src/core/datetime.h src/core/optional.h \ jeeps/gpsmath.h jeeps/gpsport.h @@ -917,14 +917,12 @@ smplrout.o: smplrout.cc defs.h config.h zlib/zlib.h zlib/zconf.h cet.h \ sort.o: sort.cc defs.h config.h zlib/zlib.h zlib/zconf.h cet.h inifile.h \ gbfile.h session.h src/core/datetime.h src/core/optional.h \ filterdefs.h filter.h sort.h +src/core/textstream.o: src/core/textstream.cc src/core/textstream.h \ + src/core/file.h defs.h config.h zlib/zlib.h zlib/zconf.h cet.h \ + inifile.h gbfile.h session.h src/core/datetime.h src/core/optional.h src/core/usasciicodec.o: src/core/usasciicodec.cc src/core/usasciicodec.h src/core/xmlstreamwriter.o: src/core/xmlstreamwriter.cc \ src/core/xmlstreamwriter.h -src/core/ziparchive.o: src/core/ziparchive.cc src/core/ziparchive.h \ - defs.h config.h zlib/zlib.h zlib/zconf.h cet.h inifile.h gbfile.h \ - session.h src/core/datetime.h src/core/optional.h \ - zlib/contrib/minizip/zip.h zlib/contrib/minizip/ioapi.h \ - src/core/logging.h stackfilter.o: stackfilter.cc defs.h config.h zlib/zlib.h zlib/zconf.h \ cet.h inifile.h gbfile.h session.h src/core/datetime.h \ src/core/optional.h filterdefs.h filter.h stackfilter.h @@ -971,11 +969,11 @@ transform.o: transform.cc defs.h config.h zlib/zlib.h zlib/zconf.h cet.h \ filterdefs.h filter.h transform.h unicsv.o: unicsv.cc defs.h config.h zlib/zlib.h zlib/zconf.h cet.h \ inifile.h gbfile.h session.h src/core/datetime.h src/core/optional.h \ - cet_util.h csv_util.h garmin_fs.h jeeps/gps.h jeeps/gpsport.h \ - jeeps/gpsdevice.h jeeps/gpssend.h jeeps/gpsread.h jeeps/gpsutil.h \ - jeeps/gpsapp.h jeeps/gpsprot.h jeeps/gpscom.h jeeps/gpsfmt.h \ - jeeps/gpsmath.h jeeps/gpsmem.h jeeps/gpsrqst.h garmin_tables.h \ - src/core/logging.h + csv_util.h garmin_fs.h jeeps/gps.h jeeps/gpsport.h jeeps/gpsdevice.h \ + jeeps/gpssend.h jeeps/gpsread.h jeeps/gpsutil.h jeeps/gpsapp.h \ + jeeps/gpsprot.h jeeps/gpscom.h jeeps/gpsfmt.h jeeps/gpsmath.h \ + jeeps/gpsmem.h jeeps/gpsrqst.h garmin_tables.h src/core/logging.h \ + src/core/textstream.h src/core/file.h units.o: units.cc defs.h config.h zlib/zlib.h zlib/zconf.h cet.h \ inifile.h gbfile.h session.h src/core/datetime.h src/core/optional.h util.o: util.cc defs.h config.h zlib/zlib.h zlib/zconf.h cet.h inifile.h \ diff --git a/csv_util.cc b/csv_util.cc index ee73344e8..58eb84f1e 100644 --- a/csv_util.cc +++ b/csv_util.cc @@ -31,6 +31,7 @@ #include "defs.h" #include "csv_util.h" +#include "src/core/logging.h" #define MYNAME "CSV_UTIL" @@ -110,7 +111,7 @@ csv_stringtrim(const char* string, const char* enclosure, int strip_max) return (tmp); } -// Is this really the replacement for the above? +// Is this really the replacement for the above? No. QString csv_stringtrim(const QString& source, const QString& enclosure) { @@ -119,6 +120,90 @@ csv_stringtrim(const QString& source, const QString& enclosure) return r.trimmed(); } +// csv_stringtrim() - trim whitespace and leading and trailing +// enclosures (quotes) +// returns a copy of the modified string +// usage: p = csv_stringtrim(string, "\"", 0) +QString +csv_stringtrim(const QString& string, const QString& enclosure, int strip_max) +{ + if (string.isEmpty()) { + return string; + } + + int elen = enclosure.size(); + + /* trim off leading and trailing whitespace */ + QString retval = string.trimmed(); + + /* if no maximum strippage, assign a reasonable value to max */ + if (strip_max == 0) { + strip_max = 9999; + } + + /* if we have enclosures, skip past them in pairs */ + if (elen > 0) { + int stripped = 0; + while ( + (stripped < strip_max) && + (retval.size() >= (elen * 2)) && + (retval.startsWith(enclosure)) && + (retval.endsWith(enclosure))) { + retval = retval.mid(elen, retval.size() - (elen * 2)); + stripped++; + } + } + + return retval; +} + +// RFC4180 method, but we don't handle line breaks within a field. +// for enclosure = " +// make str = blank into nothing +// make str = foo into "foo" +// make str = foo"bar into "foo""bar" +// No, that doesn't seem obvious to me, either... + +QString +csv_enquote(const QString& str, const QString& enclosure) +{ + QString retval = str; + if (enclosure.size() > 0) { + retval = enclosure + retval.replace(enclosure, enclosure + enclosure) + enclosure; + } + return retval; +} + +// RFC4180 method, but we don't handle line breaks within a field, +// and we strip spaces from the ends. +// csv_dequote() - trim whitespace, leading and trailing +// enclosures (quotes), and de-escape +// internal enclosures. +// usage: p = csv_dequote(string, "\"") +QString +csv_dequote(const QString& string, const QString& enclosure) +{ + if (string.isEmpty()) { + return string; + } + + int elen = enclosure.size(); + + /* trim off leading and trailing whitespace */ + QString retval = string.trimmed(); + + if (elen > 0) { + /* If the string is enclosed in enclosures */ + if (retval.startsWith(enclosure) && retval.endsWith(enclosure)) { + /* strip the enclosures */ + retval = retval.mid(elen, retval.size() - (elen * 2)); + /* replace any contained escaped enclosures */ + retval = retval.replace(enclosure + enclosure, enclosure); + } + } + + return retval; +} /*****************************************************************************/ /* csv_lineparse() - extract data fields from a delimited string. designed */ /* to handle quoted and delimited data within quotes. */ @@ -235,6 +320,102 @@ csv_lineparse(const char* stringstart, const char* delimited_by, return (tmp); } +/*****************************************************************************/ +/* csv_linesplit() - extract data fields from a delimited string. designed */ +/* to handle quoted and delimited data within quotes. */ +/* usage: p = csv_lineparse(string, ",", "\"", line) */ +/*****************************************************************************/ +QStringList +csv_linesplit(const QString& string, const QString& delimited_by, + const QString& enclosed_in, const int line_no, CsvQuoteMethod method) +{ + QStringList retval; + + const bool hyper_whitespace_delimiter = delimited_by == "\\w"; + + /* + * This is tacky. Our "csv" format is actually "commaspace" format. + * Changing that causes unwanted churn, but it also makes "real" + * comma separated data (such as likely to be produced by Excel, etc.) + * unreadable. So we silently change it here on a read and let the + * whitespace eater consume the space. + */ + QString delimiter = delimited_by; + if (delimited_by == ", ") { + delimiter = ","; + } + + /* length of delimiters and enclosures */ + int dlen = 0; + if ((!delimiter.isEmpty()) && (!hyper_whitespace_delimiter)) { + dlen = delimiter.size(); + } + int elen = enclosed_in.size(); + + int p = 0; + bool endofline = false; + while (!endofline) { + bool efound = false; + bool dfound = false; + bool enclosed = false; + + /* the beginning of the string we start with (this pass) */ + const int sp = p; + + while (p < string.size() && !dfound) { + if ((elen > 0) && string.midRef(p).startsWith(enclosed_in)) { + efound = true; + p += elen; + enclosed = !enclosed; + continue; + } + + if (!enclosed) { + if ((dlen > 0) && string.midRef(p).startsWith(delimiter)) { + dfound = true; + } else if (hyper_whitespace_delimiter && string.at(p).isSpace()) { + dfound = true; + while ((p < string.size()) && string.at(p).isSpace()) { + p++; + } + } else { + p++; + } + } else { + p++; + } + } + + QString value = string.mid(sp, p - sp); + + if (efound) { + if (method == CsvQuoteMethod::rfc4180) { + value = csv_dequote(value, enclosed_in); + } else { + value = csv_stringtrim(value, enclosed_in, 0); + } + } + + if (dfound) { + /* skip over the delimiter */ + p += dlen; + } else { + endofline = true; + } + + if (enclosed) { + Warning() << MYNAME":" << + "Warning- Unbalanced Field Enclosures" << + enclosed_in << + "on line" << + line_no; + } + + retval.append(value); + + } + return retval; +} /*****************************************************************************/ /* dec_to_intdeg() - convert decimal degrees to integer degreees */ /* usage: i = dec_to_intdeg(31.1234); */ diff --git a/csv_util.h b/csv_util.h index abcd2d2a5..b63155bf0 100644 --- a/csv_util.h +++ b/csv_util.h @@ -34,9 +34,20 @@ char* csv_stringtrim(const char* string, const char* enclosure, int strip_max); QString csv_stringtrim(const QString& source, const QString& enclosure); +QString +csv_stringtrim(const QString& string, const QString& enclosure, int strip_max); +QString +csv_enquote(const QString& str, const QString& enclosure); +QString +csv_dequote(const QString& string, const QString& enclosure); + +enum class CsvQuoteMethod {historic, rfc4180}; char* csv_lineparse(const char* stringstart, const char* delimited_by, const char* enclosed_in, int line_no); +QStringList +csv_linesplit(const QString& string, const QString& delimited_by, + const QString& enclosed_in, const int line_no, CsvQuoteMethod method = CsvQuoteMethod::historic); int dec_to_intdeg(const double d); diff --git a/defs.h b/defs.h index b817042b9..431c233b0 100644 --- a/defs.h +++ b/defs.h @@ -1110,7 +1110,6 @@ inline int case_ignore_strncmp(const QString& s1, const QString& s2, int n) } int str_match(const char* str, const char* match); -QString strenquote(const QString& str, QChar quot_char); char* strsub(const char* s, const char* search, const char* replace); char* gstrsub(const char* s, const char* search, const char* replace); @@ -1233,6 +1232,8 @@ void gb_setbit(void* buf, uint32_t nr); void* gb_int2ptr(int i); int gb_ptr2int(const void* p); +void list_codecs(); + /* * From parse.c */ diff --git a/main.cc b/main.cc index 159098b29..a8c416382 100644 --- a/main.cc +++ b/main.cc @@ -45,7 +45,7 @@ #include "defs.h" #include "cet_util.h" // for cet_convert_init, cet_convert_strings, cet_convert_deinit, cet_deregister, cet_register, cet_cs_vec_utf8 -#include "csv_util.h" // for csv_lineparse +#include "csv_util.h" // for csv_linesplit #include "filter.h" // for Filter #include "filterdefs.h" // for disp_filter_vec, disp_filter_vecs, disp_filters, exit_filter_vecs, find_filter_vec, free_filter_vec, init_filter_vecs #include "inifile.h" // for inifile_done, inifile_init @@ -102,21 +102,9 @@ load_args(const QString& filename, const QString& arg0) } file.close(); - // We use csv_lineparse to protect quoted strings, otherwise - // we could just split on blank and eliminate the round trip - // to 8 bit characters and back. - // TODO: move csv processing to Qt, eliminating the need to go - // back to 8 bit encoding, which is shaky for encoding like utf8 - // that have multibyte characters. - char* cbuff = xstrdup(CSTR(line)); - - char* cstr = csv_lineparse(cbuff, " ", "\"", 0); - while (cstr != nullptr) { - qargs.append(QString::fromUtf8(cstr)); - cstr = csv_lineparse(nullptr, " ", "\"", 0); - } + const QStringList values = csv_linesplit(line, " ", "\"", 0); + qargs.append(values); - xfree(cbuff); return (qargs); } diff --git a/ozi.cc b/ozi.cc index 8c3015cb9..4baada56b 100644 --- a/ozi.cc +++ b/ozi.cc @@ -36,29 +36,27 @@ */ -#include // for tolower -#include // for lround -#include // for atoi - -#include // for QByteArray -#include // for operator==, QChar -#include // for QCharRef -#include // for QFile -#include // for QFileInfo -#include // for QFlags -#include // for operator|, QIODevice::WriteOnly, QIODevice::ReadOnly, QIODevice, QIODevice::OpenModeFlag -#include // for QString -#include // for QStringList -#include // for QTextCodec -#include // for QTextStream, operator<<, qSetRealNumberPrecision, QTextStream::FixedNotation -#include // for CaseInsensitive -#include // for qPrintable +#include // for tolower +#include // for lround +#include // for atoi + +#include // for QByteArray +#include // for operator==, QChar +#include // for QCharRef +#include // for QFile +#include // for QFileInfo +#include // for operator|, QIODevice::WriteOnly, QIODevice::ReadOnly, QIODevice, QIODevice::OpenModeFlag +#include // for QString +#include // for QStringList +#include // for QTextStream, operator<<, qSetRealNumberPrecision, QTextStream::FixedNotation +#include // for CaseInsensitive +#include // for qPrintable #include "defs.h" -#include "csv_util.h" // for csv_stringclean -#include "jeeps/gpsmath.h" // for GPS_Math_Known_Datum_To_WGS84_M -#include "src/core/datetime.h" // for DateTime -#include "src/core/file.h" // for File +#include "csv_util.h" // for csv_stringclean +#include "jeeps/gpsmath.h" // for GPS_Math_Known_Datum_To_WGS84_M +#include "src/core/datetime.h" // for DateTime +#include "src/core/textstream.h" // for TextStream #define MYNAME "OZI" @@ -71,11 +69,7 @@ struct ozi_fsdata { int bgcolor; }; -static struct { - gpsbabel::File* file{nullptr}; - QTextStream* stream{nullptr}; - QTextCodec* codec{nullptr}; -} ozi_file; +static gpsbabel::TextStream* stream = nullptr; static short_handle mkshort_handle; static route_head* trk_head; @@ -160,36 +154,20 @@ static QString ozi_ofname; static void ozi_open_io(const QString& fname, QIODevice::OpenModeFlag mode) { - ozi_file.codec = QTextCodec::codecForName(opt_codec); - if (ozi_file.codec == nullptr) { - fatal(MYNAME ": Unsupported character set '%s'.\n", opt_codec); - } - - ozi_file.file = new gpsbabel::File(fname); - ozi_file.file->open(mode); - ozi_file.stream = new QTextStream(ozi_file.file); - ozi_file.stream->setCodec(ozi_file.codec); - - if (mode | QFile::WriteOnly) { - ozi_file.stream->setRealNumberNotation(QTextStream::FixedNotation); - } + stream = new gpsbabel::TextStream; + stream->open(fname, mode, MYNAME, opt_codec); - if (mode | QFile::ReadOnly) { - if (ozi_file.codec->mibEnum() == 106) { // UTF-8 - ozi_file.stream->setAutoDetectUnicode(true); - } + if (mode & QFile::WriteOnly) { + stream->setRealNumberNotation(QTextStream::FixedNotation); } } static void ozi_close_io() { - ozi_file.file->close(); - delete ozi_file.file; - ozi_file.file = nullptr; - delete ozi_file.stream; - ozi_file.stream = nullptr; - ozi_file.codec = nullptr; + stream->close(); + delete stream; + stream = nullptr; } static void @@ -267,7 +245,7 @@ ozi_openfile(const QString& fname) */ if (fname == "-") { - if (ozi_file.file == nullptr) { + if (stream == nullptr) { ozi_open_io(fname, QFile::WriteOnly); } return; @@ -291,7 +269,7 @@ ozi_openfile(const QString& fname) QString tmpname = QString("%1%2.%3").arg(sname, buff, ozi_extensions[ozi_objective]); /* re-open file_out with the new filename */ - if (ozi_file.file != nullptr) { + if (stream != nullptr) { ozi_close_io(); } @@ -303,7 +281,7 @@ ozi_track_hdr(const route_head* rte) { if ((! pack_opt) || (track_out_count == 0)) { ozi_openfile(ozi_ofname); - *ozi_file.stream << "OziExplorer Track Point File Version 2.1\r\n" + *stream << "OziExplorer Track Point File Version 2.1\r\n" << "WGS 84\r\n" << "Altitude is in " << (altunit == 'f' ? "Feet" : "Meters") << "\r\n" << "Reserved 3\r\n" @@ -330,7 +308,7 @@ ozi_track_disp(const Waypoint* waypointp) alt = waypointp->altitude * alt_scale; } - *ozi_file.stream << qSetRealNumberPrecision(6) << waypointp->latitude << ',' + *stream << qSetRealNumberPrecision(6) << waypointp->latitude << ',' << waypointp->longitude << ',' << new_track << ',' << qSetRealNumberPrecision(0) << alt << ',' @@ -350,7 +328,7 @@ ozi_route_hdr(const route_head* rte) { /* prologue on 1st pass only */ if (route_out_count == 0) { - *ozi_file.stream << "OziExplorer Route File Version 1.0\r\n" + *stream << "OziExplorer Route File Version 1.0\r\n" << "WGS 84\r\n" << "Reserved 1\r\n" << "Reserved 2\r\n"; @@ -371,7 +349,7 @@ ozi_route_hdr(const route_head* rte) * R, 1, ICP GALHETA,, 16711680 */ - *ozi_file.stream << "R," << route_out_count << ',' + *stream << "R," << route_out_count << ',' << rte->rte_name << ',' << rte->rte_desc << ",\r\n"; } @@ -412,7 +390,7 @@ ozi_route_disp(const Waypoint* waypointp) * W,1,7,7,007,-25.581670,-48.316660,36564.54196,10,1,4,0,65535,TR ILHA GALHETA,0,0 */ - *ozi_file.stream << "W," << route_out_count << ",," + *stream << "W," << route_out_count << ",," << route_wpt_count << ',' << waypointp->shortname << ',' << qSetRealNumberPrecision(6) << waypointp->latitude << ',' @@ -765,11 +743,7 @@ data_read() char* trk_name = nullptr; int linecount = 0; - while (true) { - buff = ozi_file.stream->readLine(); - if (buff.isNull()) { - break; - } + while (buff = stream->readLine(), !buff.isNull()) { linecount++; /* @@ -950,7 +924,7 @@ ozi_waypt_pr(const Waypoint* wpt) icon = wpt->icon_descr.toInt(); } - *ozi_file.stream << index << ',' + *stream << index << ',' << shortname << ',' << qSetRealNumberPrecision(6) << wpt->latitude << ',' << wpt->longitude << ',' @@ -961,13 +935,13 @@ ozi_waypt_pr(const Waypoint* wpt) << fs->bgcolor << ',' << description << ",0,0,"; if (WAYPT_HAS(wpt, proximity) && (wpt->proximity > 0)) { - *ozi_file.stream << qSetRealNumberPrecision(1) << wpt->proximity * prox_scale << ','; + *stream << qSetRealNumberPrecision(1) << wpt->proximity * prox_scale << ','; } else if (proximity > 0) { - *ozi_file.stream << qSetRealNumberPrecision(1) << proximity * prox_scale << ','; + *stream << qSetRealNumberPrecision(1) << proximity * prox_scale << ','; } else { - *ozi_file.stream << "0,"; + *stream << "0,"; } - *ozi_file.stream << qSetRealNumberPrecision(0) << alt << ",6,0,17\r\n"; + *stream << qSetRealNumberPrecision(0) << alt << ",6,0,17\r\n"; if (faked_fsdata) { xfree(fs); @@ -981,7 +955,7 @@ data_write() track_out_count = route_out_count = 0; ozi_objective = wptdata; ozi_openfile(ozi_ofname); - *ozi_file.stream << "OziExplorer Waypoint File Version 1.1\r\n" + *stream << "OziExplorer Waypoint File Version 1.1\r\n" << "WGS 84\r\n" << "Reserved 2\r\n" << "Reserved 3\r\n"; diff --git a/reference/libreoffice.csv b/reference/libreoffice.csv new file mode 100755 index 000000000..8cac45a0e --- /dev/null +++ b/reference/libreoffice.csv @@ -0,0 +1,2 @@ +No,Latitude,Longitude,Name,Description +1,40.015,-105.2705,"Boulder, Colorado","In ""Where I Lived, and What I Lived For,"" Thoreau states directly his purpose for going into the woods: ""I went to the woods because I wished to live deliberately, to front only the essential facts of life, and see if I could not learn what it had to teach, and not, when I came to die, discover that I had not lived.""" diff --git a/reference/libreoffice.text b/reference/libreoffice.text new file mode 100644 index 000000000..fa47ee013 --- /dev/null +++ b/reference/libreoffice.text @@ -0,0 +1,4 @@ +----------------------------------------------------------------------------- +Boulder, Colorado N40.01500 W105.27050 (13T 476915 4429457) +In "Where I Lived, and What I Lived For," Thoreau states directly his purpose for going into the woods: "I went to the woods because I wished to live deliberately, to front only the essential facts of life, and see if I could not learn what it had to teach, and not, when I came to die, discover that I had not lived." +----------------------------------------------------------------------------- diff --git a/src/core/textstream.cc b/src/core/textstream.cc new file mode 100644 index 000000000..6106621aa --- /dev/null +++ b/src/core/textstream.cc @@ -0,0 +1,68 @@ +/* + Copyright (C) 2019 Robert Lipe, gpsbabel.org + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111 USA + + */ + +#include // for QFile + +#include "src/core/textstream.h" +#include "defs.h" // for fatal +#include "src/core/file.h" // for File + + +namespace gpsbabel +{ + +void TextStream::open(const QString& fname, QIODevice::OpenModeFlag mode, const char* module, const char* codec_name) +{ + codec_ = QTextCodec::codecForName(codec_name); + if (codec_ == nullptr) { + list_codecs(); + fatal("%s: Unsupported codec '%s'.\n", module, codec_name); + } + + file_ = new gpsbabel::File(fname); + file_->open(mode); + setDevice(file_); + setCodec(codec_); + + if (mode & QFile::ReadOnly) { + if (codec_->mibEnum() == 106) { // UTF-8 + setAutoDetectUnicode(true); + } + } + + if (mode & QFile::WriteOnly) { + // enable bom for all UTF codecs except UTF-8 + if (codec_->mibEnum() != 106) { + setGenerateByteOrderMark(true); + } + } +} + +void TextStream::close() +{ + flush(); + if (file_ != nullptr) { + file_->close(); + delete file_; + file_ = nullptr; + } + codec_ = nullptr; +} + +}; // namespace diff --git a/src/core/textstream.h b/src/core/textstream.h new file mode 100644 index 000000000..463e10ec9 --- /dev/null +++ b/src/core/textstream.h @@ -0,0 +1,43 @@ +/* + Copyright (C) 2019 Robert Lipe, gpsbabel.org + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111 USA + + */ + +#include // for QByteArray +#include // for QIODevice, QIODevice::OpenModeFlag +#include // for QString +#include // for QTextCodec +#include // for QTextStream + +#include "src/core/file.h" // for File + + +namespace gpsbabel +{ + +class TextStream : public QTextStream +{ +public: + void open(const QString& fname, QIODevice::OpenModeFlag mode, const char* module, const char* codec = "UTF-8"); + void close(); + +private: + gpsbabel::File* file_{nullptr}; + QTextCodec* codec_{nullptr}; +}; + +}; // namespace diff --git a/testo.d/unicsv.test b/testo.d/unicsv.test index 702574e06..96140a437 100644 --- a/testo.d/unicsv.test +++ b/testo.d/unicsv.test @@ -30,3 +30,11 @@ compare ${REFERENCE}/unicsv_subsec.csv ${TMPDIR}/unicsv_subsec.csv # Verify 'fields' option gpsbabel -i unicsv,fields=lat+lon+description -f ${REFERENCE}/radius.csv -o csv -F ${TMPDIR}/unicsv_fields.out compare ${REFERENCE}/radius.csv ${TMPDIR}/unicsv_fields.out + +# stress quoting - internal separators and internal quotes +gpsbabel -i unicsv -f ${REFERENCE}/libreoffice.csv -o text,degformat=ddd -F ${TMPDIR}/libreoffice.text +compare ${REFERENCE}/libreoffice.text ${TMPDIR}/libreoffice.text + +gpsbabel -i unicsv -f ${REFERENCE}/libreoffice.csv -o unicsv -F ${TMPDIR}/libreoffice2.csv +gpsbabel -i unicsv -f ${TMPDIR}/libreoffice2.csv -o text,degformat=ddd -F ${TMPDIR}/libreoffice2.text +compare ${REFERENCE}/libreoffice.text ${TMPDIR}/libreoffice2.text diff --git a/unicsv.cc b/unicsv.cc index 9e9067674..731b5a68f 100644 --- a/unicsv.cc +++ b/unicsv.cc @@ -19,16 +19,38 @@ Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111 USA */ +#include // for fabs, lround +#include // for NULL, sscanf +#include +#include // for atoi +#include // for memset, strchr, strncpy +#include // for gmtime + +#include // for QByteArray +#include // for QChar +#include // for QCharRef +#include // for QDateTime +#include // for QIODevice, QIODevice::ReadOnly, QIODevice::WriteOnly +#include // for QLatin1Char +#include // for QLatin1String +#include // for QString, operator!=, operator== +#include // for QStringList +#include // for QTextStream, operator<<, qSetRealNumberPrecision, qSetFieldWidth, QTextStream::FixedNotation +#include // for QTime +#include // for QVector +#include // for CaseInsensitive +#include // for qPrintable + #include "defs.h" -#include "cet.h" -#include "cet_util.h" -#include "csv_util.h" -#include "garmin_fs.h" -#include "garmin_tables.h" -#include "jeeps/gpsmath.h" -#include "src/core/logging.h" -#include -#include +#include "csv_util.h" // for csv_linesplit, human_to_dec +#include "garmin_fs.h" // for garmin_fs_flags_t, garmin_fs_t, GMSD_GET, GMSD_HAS, GMSD_SETQSTR, GMSD_FIND, garmin_fs_alloc +#include "garmin_tables.h" // for gt_lookup_datum_index, gt_get_mps_grid_longname, gt_lookup_grid_type +#include "jeeps/gpsmath.h" // for GPS_Math_UKOSMap_To_WGS84_M, GPS_Math_EN_To_UKOSNG_Map, GPS_Math_Known_Datum_To_UTM_EN, GPS_Math_Known_Datum_To_WGS84_M, GPS_Math_Swiss_EN_To_WGS84, GPS_Math_UTM_EN_To_Known_Datum, GPS_Math_WGS84_To_Known_Datum_M, GPS_Math_WGS84_To_Swiss_EN, GPS_Math_WGS... +#include "session.h" // for session_t +#include "src/core/datetime.h" // for DateTime +#include "src/core/logging.h" // for Warning, Fatal +#include "src/core/textstream.h" // for TextStream + #define MYNAME "unicsv" @@ -36,7 +58,7 @@ #define UNICSV_FIELD_SEP "," #define UNICSV_LINE_SEP "\r\n" -#define UNICSV_QUOT_CHAR '"' +#define UNICSV_QUOT_CHAR "\"" /* GPSBabel internal and calculated fields */ @@ -244,7 +266,8 @@ static QVector unicsv_fields_tab; static double unicsv_altscale, unicsv_depthscale, unicsv_proximityscale ; static const char* unicsv_fieldsep; -static gbfile* fin, *fout; +static gpsbabel::TextStream* fin = nullptr; +static gpsbabel::TextStream* fout = nullptr; static gpsdata_type unicsv_data_type; static route_head* unicsv_track, *unicsv_route; static char unicsv_outp_flags[(fld_terminator + 8) / 8]; @@ -257,6 +280,7 @@ static char* opt_filename; static char* opt_format; static char* opt_prec; static char* opt_fields; +static char* opt_codec; static int unicsv_waypt_ct; static char unicsv_detect; static int llprec; @@ -290,25 +314,16 @@ static arglist_t unicsv_args[] = { "fields", &opt_fields, "Name and order of input fields, separated by '+'", nullptr, ARGTYPE_STRING, ARG_NOMINMAX, nullptr }, + { + "codec", &opt_codec, "codec to use for reading and writing strings (default UTF-8)", + "UTF-8", ARGTYPE_STRING, ARG_NOMINMAX, nullptr + }, ARG_TERMINATOR }; /* helpers */ -/* here we only need a simple yes(0) or no(1) */ -static int -unicsv_strrcmp(const char* s1, const char* s2) -{ - int l1 = strlen(s1); - int l2 = strlen(s2); - if ((l1 - l2) >= 0) { - return strcmp(s1 + (l1 - l2), s2); - } else { - return 1; /* false */ - } -} - // There is no test coverage of this and it's been wrong for years and // nobody has noticed... static int @@ -457,139 +472,102 @@ unicsv_adjust_time(const time_t time, const time_t* date) return QDateTime::fromTime_t(res); } -static char -unicsv_compare_fields(const char* s, const field_t* f) +static bool +unicsv_compare_fields(const QString& s, const field_t* f) { - const char* name = f->name; - const char* test = s; - char result; + QString name = f->name; + QString test = s; + bool result = false; if (!(f->options & STR_CASE)) { - test = strupper(xstrdup(s)); - name = strupper(xstrdup(f->name)); + test = test.toUpper(); + name = name.toUpper(); } if (f->options & STR_EQUAL) { - result = (strcmp(test, name) == 0); + result = test == name; } else if (f->options & STR_ANY) { - result = (strstr(test, name) != nullptr); - } else { - if (f->options & STR_LEFT) { - result = (strncmp(test, name, strlen(name)) == 0); - } else if (f->options & STR_RIGHT) { - result = (unicsv_strrcmp(test, name) == 0); - } else { - result = 0; /* fallback to "FALSE" */ - } + result = test.contains(name); + } else if (f->options & STR_LEFT) { + result = test.startsWith(name); + } else if (f->options & STR_RIGHT) { + result = test.endsWith(name); } - if ((! result) && (strchr(test, ' ') != nullptr)) { + if ((! result) && test.contains(' ')) { /* replace ' ' with '_' and try again */ - char* tmp = gstrsub(test, " ", "_"); - result = unicsv_compare_fields(tmp, f); - xfree(tmp); + result = unicsv_compare_fields(test.replace(' ', '_'), f); } - if ((! result) && (strchr(test, '-') != nullptr)) { + if ((! result) && test.contains('-')) { /* replace '-' with '_' and try again */ - char* tmp = gstrsub(test, "-", "_"); - result = unicsv_compare_fields(tmp, f); - xfree(tmp); - } - - if (name != f->name) { - xfree(name); - xfree(test); + result = unicsv_compare_fields(test.replace('-', '_'), f); } return result; } -static char -unicsv_compare_fields(const QString& s, const field_t* f) -{ - return unicsv_compare_fields(CSTR(s), f); -} - static void -unicsv_fondle_header(QString s) +unicsv_fondle_header(QString header) { - // TODO: clean up this back and forth between QString and char*. - char* buf = nullptr; - const cet_cs_vec_t* ascii = &cet_cs_vec_ansi_x3_4_1968; /* us-ascii */ - /* Convert the entire header to lower case for convenience. * If we see a tab in that header, we decree it to be tabsep. */ unicsv_fieldsep = ","; - if (s.contains('\t')) { + if (header.contains('\t')) { unicsv_fieldsep = "\t"; - } else if (s.contains(';')) { + } else if (header.contains(';')) { unicsv_fieldsep = ";"; - } else if (s.contains('|')) { + } else if (header.contains('|')) { unicsv_fieldsep = "|"; } - char* cbuf_start = xstrdup(s.toLower()); - const char* cbuf = cbuf_start; + header = header.toLower(); - /* convert the header line into native ascii */ - if (global_opts.charset != ascii) { - buf = cet_str_any_to_any(cbuf, global_opts.charset, ascii); - cbuf = buf; - } - - while ((s = csv_lineparse(cbuf, unicsv_fieldsep, "\"", 0)), !s.isNull()) { - s = s.trimmed(); + const QStringList values = csv_linesplit(header, unicsv_fieldsep, "\"", 0, CsvQuoteMethod::rfc4180); + for (auto value : values) { + value = value.trimmed(); field_t* f = &fields_def[0]; - cbuf = nullptr; - unicsv_fields_tab.append(fld_terminator); while (f->name) { - if (unicsv_compare_fields(s, f)) { + if (unicsv_compare_fields(value, f)) { unicsv_fields_tab.last() = f->type; break; } f++; } if ((! f->name) && global_opts.debug_level) { - warning(MYNAME ": Unhandled column \"%s\".\n", qPrintable(s)); + warning(MYNAME ": Unhandled column \"%s\".\n", qPrintable(value)); } /* handle some special items */ if (f->type == fld_altitude) { - if (s.contains("ft") || s.contains("feet")) { + if (value.contains("ft") || value.contains("feet")) { unicsv_altscale = FEET_TO_METERS(1); } } if (f->type == fld_depth) { - if (s.contains("ft") || s.contains("feet")) { + if (value.contains("ft") || value.contains("feet")) { unicsv_depthscale = FEET_TO_METERS(1); } } if (f->type == fld_proximity) { - if (s.contains("ft") || s.contains("feet")) { + if (value.contains("ft") || value.contains("feet")) { unicsv_proximityscale = FEET_TO_METERS(1); } } if ((f->type == fld_time) || (f->type == fld_date)) { - if (s.contains("iso")) { + if (value.contains("iso")) { f->type = fld_iso_time; } } } - if (buf) { - xfree(buf); - } - if (cbuf_start) { - xfree(cbuf_start); - } } static void unicsv_rd_init(const QString& fname) { - char* c; + QString buff; unicsv_altscale = 1.0; unicsv_depthscale = 1.0; unicsv_proximityscale = 1.0; @@ -601,29 +579,29 @@ unicsv_rd_init(const QString& fname) unicsv_track = unicsv_route = nullptr; unicsv_datum_idx = gt_lookup_datum_index(opt_datum, MYNAME); - fin = gbfopen(fname, "rb", MYNAME); + fin = new gpsbabel::TextStream; + fin->open(fname, QIODevice::ReadOnly, MYNAME, opt_codec); if (opt_fields) { QString fields = QString(opt_fields).replace("+", ","); unicsv_fondle_header(fields); - } else if ((c = gbfgetstr(fin))) { - unicsv_fondle_header(c); + } else if (buff = fin->readLine(), !buff.isNull()) { + unicsv_fondle_header(buff); } else { unicsv_fieldsep = nullptr; } - if (fin->unicode) { - cet_convert_init(CET_CHARSET_UTF8, 1); - } } static void unicsv_rd_deinit() { - gbfclose(fin); + fin->close(); + delete fin; + fin = nullptr; unicsv_fields_tab.clear(); } static void -unicsv_parse_one_line(char* ibuf) +unicsv_parse_one_line(const QString& ibuf) { int utm_zone = -9999; double utm_easting = 0; @@ -653,17 +631,15 @@ unicsv_parse_one_line(char* ibuf) memset(&ymd, 0, sizeof(ymd)); int column = -1; - QString s; - while ((s = csv_lineparse(ibuf, unicsv_fieldsep, "\"", 0)), !s.isNull()) { + const QStringList values = csv_linesplit(ibuf, unicsv_fieldsep, "\"", 0, CsvQuoteMethod::rfc4180); + for (auto value : values) { if (++column >= unicsv_fields_tab.size()) { break; /* ignore extra fields on line */ } - ibuf = nullptr; - checked++; - s = s.trimmed(); - if (s.isEmpty()) { + value = value.trimmed(); + if (value.isEmpty()) { continue; /* skip empty columns */ } switch (unicsv_fields_tab[column]) { @@ -672,7 +648,7 @@ unicsv_parse_one_line(char* ibuf) case fld_date: case fld_datetime: /* switch column type if it looks like an iso time string */ - if (s.contains('T')) { + if (value.contains('T')) { unicsv_fields_tab[column] = fld_iso_time; } break; @@ -684,34 +660,34 @@ unicsv_parse_one_line(char* ibuf) switch (unicsv_fields_tab[column]) { case fld_latitude: - human_to_dec(CSTR(s), &wpt->latitude, nullptr, 1); + human_to_dec(CSTR(value), &wpt->latitude, nullptr, 1); wpt->latitude = wpt->latitude * ns; break; case fld_longitude: - human_to_dec(CSTR(s), nullptr, &wpt->longitude, 2); + human_to_dec(CSTR(value), nullptr, &wpt->longitude, 2); wpt->longitude = wpt->longitude * ew; break; case fld_shortname: - wpt->shortname = s; + wpt->shortname = value; break; case fld_description: - wpt->description = s; + wpt->description = value; break; case fld_notes: - wpt->notes = s; + wpt->notes = value; break; case fld_url: { - wpt->AddUrlLink(s); + wpt->AddUrlLink(value); } break; case fld_altitude: - if (parse_distance(s, &d, unicsv_altscale, MYNAME)) { + if (parse_distance(value, &d, unicsv_altscale, MYNAME)) { if (fabs(d) < fabs(unknown_alt)) { wpt->altitude = d; } @@ -719,23 +695,23 @@ unicsv_parse_one_line(char* ibuf) break; case fld_utm_zone: - utm_zone = s.toInt(); + utm_zone = value.toInt(); break; case fld_utm_easting: - utm_easting = s.toDouble(); + utm_easting = value.toDouble(); break; case fld_utm_northing: - utm_northing = s.toDouble(); + utm_northing = value.toDouble(); break; case fld_utm_zone_char: - utm_zc = s[0].toUpper().toLatin1(); + utm_zc = value.at(0).toUpper().toLatin1(); break; case fld_utm: - parse_coordinates(s, unicsv_datum_idx, grid_utm, + parse_coordinates(value, unicsv_datum_idx, grid_utm, &wpt->latitude, &wpt->longitude, MYNAME); /* coordinates from parse_coordinates are in WGS84 don't convert a second time */ @@ -743,7 +719,7 @@ unicsv_parse_one_line(char* ibuf) break; case fld_bng: - parse_coordinates(s, DATUM_OSGB36, grid_bng, + parse_coordinates(value, DATUM_OSGB36, grid_bng, &wpt->latitude, &wpt->longitude, MYNAME); /* coordinates from parse_coordinates are in WGS84 don't convert a second time */ @@ -751,20 +727,20 @@ unicsv_parse_one_line(char* ibuf) break; case fld_bng_zone: - strncpy(bng_zone, CSTR(s), sizeof(bng_zone) -1); + strncpy(bng_zone, CSTR(value), sizeof(bng_zone) - 1); strupper(bng_zone); break; case fld_bng_northing: - bng_northing = s.toDouble(); + bng_northing = value.toDouble(); break; case fld_bng_easting: - bng_easting = s.toDouble(); + bng_easting = value.toDouble(); break; case fld_swiss: - parse_coordinates(s, DATUM_WGS84, grid_swiss, + parse_coordinates(value, DATUM_WGS84, grid_swiss, &wpt->latitude, &wpt->longitude, MYNAME); /* coordinates from parse_coordinates are in WGS84 don't convert a second time */ @@ -772,36 +748,36 @@ unicsv_parse_one_line(char* ibuf) break; case fld_swiss_easting: - swiss_easting = s.toDouble(); + swiss_easting = value.toDouble(); break; case fld_swiss_northing: - swiss_northing = s.toDouble(); + swiss_northing = value.toDouble(); break; case fld_hdop: - wpt->hdop = s.toDouble(); + wpt->hdop = value.toDouble(); if (unicsv_detect) { unicsv_data_type = trkdata; } break; case fld_pdop: - wpt->pdop = s.toDouble(); + wpt->pdop = value.toDouble(); if (unicsv_detect) { unicsv_data_type = trkdata; } break; case fld_vdop: - wpt->vdop = s.toDouble(); + wpt->vdop = value.toDouble(); if (unicsv_detect) { unicsv_data_type = trkdata; } break; case fld_sat: - wpt->sat = s.toInt(); + wpt->sat = value.toInt(); if (unicsv_detect) { unicsv_data_type = trkdata; } @@ -811,15 +787,15 @@ unicsv_parse_one_line(char* ibuf) if (unicsv_detect) { unicsv_data_type = trkdata; } - if (case_ignore_strcmp(s, "none") == 0) { + if (case_ignore_strcmp(value, "none") == 0) { wpt->fix = fix_none; - } else if (case_ignore_strcmp(s, "2d") == 0) { + } else if (case_ignore_strcmp(value, "2d") == 0) { wpt->fix = fix_2d; - } else if (case_ignore_strcmp(s, "3d") == 0) { + } else if (case_ignore_strcmp(value, "3d") == 0) { wpt->fix = fix_3d; - } else if (case_ignore_strcmp(s, "dgps") == 0) { + } else if (case_ignore_strcmp(value, "dgps") == 0) { wpt->fix = fix_dgps; - } else if (case_ignore_strcmp(s, "pps") == 0) { + } else if (case_ignore_strcmp(value, "pps") == 0) { wpt->fix = fix_pps; } else { wpt->fix = fix_unknown; @@ -828,20 +804,20 @@ unicsv_parse_one_line(char* ibuf) case fld_utc_date: if ((is_localtime < 2) && (date < 0)) { - date = unicsv_parse_date(CSTR(s), nullptr); + date = unicsv_parse_date(CSTR(value), nullptr); is_localtime = 0; } break; case fld_utc_time: if ((is_localtime < 2) && (time < 0)) { - time = unicsv_parse_time(s, &usec, &date); + time = unicsv_parse_time(value, &usec, &date); is_localtime = 0; } break; case fld_speed: - if (parse_speed(s, &d, 1.0, MYNAME)) { + if (parse_speed(value, &d, 1.0, MYNAME)) { WAYPT_SET(wpt, speed, d); if (unicsv_detect) { unicsv_data_type = trkdata; @@ -850,120 +826,120 @@ unicsv_parse_one_line(char* ibuf) break; case fld_course: - WAYPT_SET(wpt, course, s.toDouble()); + WAYPT_SET(wpt, course, value.toDouble()); if (unicsv_detect) { unicsv_data_type = trkdata; } break; case fld_temperature: - d = s.toDouble(); + d = value.toDouble(); if (fabs(d) < 999999) { WAYPT_SET(wpt, temperature, d); } break; case fld_temperature_f: - d = s.toDouble(); + d = value.toDouble(); if (fabs(d) < 999999) { WAYPT_SET(wpt, temperature, FAHRENHEIT_TO_CELSIUS(d)); } break; case fld_heartrate: - wpt->heartrate = s.toInt(); + wpt->heartrate = value.toInt(); if (unicsv_detect) { unicsv_data_type = trkdata; } break; case fld_cadence: - wpt->cadence = s.toInt(); + wpt->cadence = value.toInt(); if (unicsv_detect) { unicsv_data_type = trkdata; } break; case fld_power: - wpt->power = s.toDouble(); + wpt->power = value.toDouble(); if (unicsv_detect) { unicsv_data_type = trkdata; } break; case fld_proximity: - if (parse_distance(s, &d, unicsv_proximityscale, MYNAME)) { + if (parse_distance(value, &d, unicsv_proximityscale, MYNAME)) { WAYPT_SET(wpt, proximity, d); } break; case fld_depth: - if (parse_distance(s, &d, unicsv_depthscale, MYNAME)) { + if (parse_distance(value, &d, unicsv_depthscale, MYNAME)) { WAYPT_SET(wpt, depth, d); } break; case fld_symbol: - wpt->icon_descr = s; + wpt->icon_descr = value; break; case fld_iso_time: is_localtime = 2; /* fix result */ - wpt->SetCreationTime(xml_parse_time(s)); + wpt->SetCreationTime(xml_parse_time(value)); break; case fld_time: if ((is_localtime < 2) && (time < 0)) { - time = unicsv_parse_time(s, &usec, &date); + time = unicsv_parse_time(value, &usec, &date); is_localtime = 1; } break; case fld_date: if ((is_localtime < 2) && (date < 0)) { - date = unicsv_parse_date(CSTR(s), nullptr); + date = unicsv_parse_date(CSTR(value), nullptr); is_localtime = 1; } break; case fld_year: - ymd.tm_year = s.toInt(); + ymd.tm_year = value.toInt(); break; case fld_month: - ymd.tm_mon = s.toInt(); + ymd.tm_mon = value.toInt(); break; case fld_day: - ymd.tm_mday = s.toInt(); + ymd.tm_mday = value.toInt(); break; case fld_hour: - ymd.tm_hour = s.toInt(); + ymd.tm_hour = value.toInt(); break; case fld_min: - ymd.tm_min = s.toInt(); + ymd.tm_min = value.toInt(); break; case fld_sec: - ymd.tm_sec = s.toInt(); + ymd.tm_sec = value.toInt(); break; case fld_datetime: if ((is_localtime < 2) && (date < 0) && (time < 0)) { - time = unicsv_parse_time(s, &usec, &date); + time = unicsv_parse_time(value, &usec, &date); is_localtime = 1; } break; case fld_ns: - ns = s.startsWith('n', Qt::CaseInsensitive) ? 1 : -1; + ns = value.startsWith('n', Qt::CaseInsensitive) ? 1 : -1; wpt->latitude *= ns; break; case fld_ew: - ew = s.startsWith('e', Qt::CaseInsensitive) ? 1 : -1; + ew = value.startsWith('e', Qt::CaseInsensitive) ? 1 : -1; wpt->longitude *= ew; break; @@ -984,34 +960,34 @@ unicsv_parse_one_line(char* ibuf) } switch (unicsv_fields_tab[column]) { case fld_garmin_city: - GMSD_SETQSTR(city, s); + GMSD_SETQSTR(city, value); break; case fld_garmin_postal_code: - GMSD_SETQSTR(postal_code, s); + GMSD_SETQSTR(postal_code, value); break; case fld_garmin_state: - GMSD_SETQSTR(state, s); + GMSD_SETQSTR(state, value); break; case fld_garmin_country: - GMSD_SETQSTR(country, s); + GMSD_SETQSTR(country, value); break; case fld_garmin_addr: - GMSD_SETQSTR(addr, s); + GMSD_SETQSTR(addr, value); break; case fld_garmin_phone_nr: - GMSD_SETQSTR(phone_nr, s); + GMSD_SETQSTR(phone_nr, value); break; case fld_garmin_phone_nr2: - GMSD_SETQSTR(phone_nr2, s); + GMSD_SETQSTR(phone_nr2, value); break; case fld_garmin_fax_nr: - GMSD_SETQSTR(fax_nr, s); + GMSD_SETQSTR(fax_nr, value); break; case fld_garmin_email: - GMSD_SETQSTR(email, s); + GMSD_SETQSTR(email, value); break; case fld_garmin_facility: - GMSD_SETQSTR(facility, s); + GMSD_SETQSTR(facility, value); break; default: break; @@ -1035,33 +1011,33 @@ unicsv_parse_one_line(char* ibuf) switch (unicsv_fields_tab[column]) { case fld_gc_id: - gc_data->id = s.toInt(); + gc_data->id = value.toInt(); if (gc_data->id == 0) { - gc_data->id = unicsv_parse_gc_id(s); + gc_data->id = unicsv_parse_gc_id(value); } break; case fld_gc_type: - gc_data->type = gs_mktype(s); + gc_data->type = gs_mktype(value); break; case fld_gc_container: - gc_data->container = gs_mkcont(s); + gc_data->container = gs_mkcont(value); break; case fld_gc_terr: - gc_data->terr = s.toDouble() * 10; + gc_data->terr = value.toDouble() * 10; break; case fld_gc_diff: - gc_data->diff = s.toDouble() * 10; + gc_data->diff = value.toDouble() * 10; break; case fld_gc_is_archived: - gc_data->is_archived = unicsv_parse_status(s); + gc_data->is_archived = unicsv_parse_status(value); break; case fld_gc_is_available: - gc_data->is_available = unicsv_parse_status(s); + gc_data->is_available = unicsv_parse_status(value); break; case fld_gc_exported: { time_t time, date; int usec; - time = unicsv_parse_time(s, &usec, &date); + time = unicsv_parse_time(value, &usec, &date); if (date || time) { gc_data->exported = unicsv_adjust_time(time, &date); } @@ -1070,20 +1046,20 @@ unicsv_parse_one_line(char* ibuf) case fld_gc_last_found: { time_t time, date; int usec; - time = unicsv_parse_time(s, &usec, &date); + time = unicsv_parse_time(value, &usec, &date); if (date || time) { gc_data->last_found = unicsv_adjust_time(time, &date); } } break; case fld_gc_placer: - gc_data->placer = s; + gc_data->placer = value; break; case fld_gc_placer_id: - gc_data->placer_id = s.toInt(); + gc_data->placer_id = value.toInt(); break; case fld_gc_hint: - gc_data->hint = s; + gc_data->hint = value; break; default: @@ -1228,15 +1204,15 @@ unicsv_parse_one_line(char* ibuf) static void unicsv_rd() { - char* buff; + QString buff; if (unicsv_fieldsep == nullptr) { return; } - while ((buff = gbfgetstr(fin))) { - buff = lrtrim(buff); - if ((*buff == '\0') || (*buff == '#')) { + while ((buff = fin->readLine(), !buff.isNull())) { + buff = buff.trimmed(); + if (buff.isEmpty() || buff.startsWith('#')) { continue; } unicsv_parse_one_line(buff); @@ -1248,7 +1224,7 @@ unicsv_rd() static void unicsv_fatal_outside(const Waypoint* wpt) { - gbfprintf(fout, "#####\n"); + *fout << "#####\n"; fatal(MYNAME ": %s (%s) is outside of convertable area of grid \"%s\"!\n", wpt->shortname.isEmpty() ? "Waypoint" : qPrintable(wpt->shortname), pretty_deg_format(wpt->latitude, wpt->longitude, 'd', nullptr, 0), @@ -1258,10 +1234,10 @@ unicsv_fatal_outside(const Waypoint* wpt) static void unicsv_print_str(const QString& s) { - gbfputs(unicsv_fieldsep, fout); + *fout << unicsv_fieldsep; QString t; if (!s.isEmpty()) { - t = strenquote(s, UNICSV_QUOT_CHAR); + t = csv_enquote(s, UNICSV_QUOT_CHAR); // I'm not sure these three replacements are necessary; they're just a // slavish re-implementation of (what I think) the original C code // was doing. @@ -1269,7 +1245,7 @@ unicsv_print_str(const QString& s) t.replace("\r", ","); t.replace("\n", ","); } - gbfputs(t.trimmed(), fout); + *fout << t.trimmed(); } static void @@ -1460,28 +1436,28 @@ unicsv_waypt_disp_cb(const Waypoint* wpt) &lat, &lon, &alt, unicsv_datum_idx); } - gbfprintf(fout, "%d%s", unicsv_waypt_ct, unicsv_fieldsep); + *fout << unicsv_waypt_ct << unicsv_fieldsep; switch (unicsv_grid_idx) { case grid_lat_lon_ddd: cout = pretty_deg_format(lat, lon, 'd', unicsv_fieldsep, 0); - gbfputs(cout, fout); + *fout << cout; break; case grid_lat_lon_dmm: cout = pretty_deg_format(lat, lon, 'm', unicsv_fieldsep, 0); - gbfputs(cout, fout); + *fout << cout; break; case grid_lat_lon_dms: { cout = pretty_deg_format(lat, lon, 's', unicsv_fieldsep, 0); char* sep = strchr(cout, ','); *sep = '\0'; - QString tmp = strenquote(cout, UNICSV_QUOT_CHAR); - gbfprintf(fout, "%s%s", CSTR(tmp), unicsv_fieldsep); - tmp = strenquote(sep+1, UNICSV_QUOT_CHAR); - gbfputs(tmp, fout); + QString tmp = csv_enquote(cout, UNICSV_QUOT_CHAR); + *fout << tmp << unicsv_fieldsep; + tmp = csv_enquote(sep+1, UNICSV_QUOT_CHAR); + *fout << tmp; } break; @@ -1492,10 +1468,11 @@ unicsv_waypt_disp_cb(const Waypoint* wpt) if (! GPS_Math_WGS84_To_UKOSMap_M(wpt->latitude, wpt->longitude, &east, &north, map)) { unicsv_fatal_outside(wpt); } - gbfprintf(fout, "%s%s%5.0f%s%5.0f", - map, unicsv_fieldsep, - east, unicsv_fieldsep, - north); + auto fieldWidth = fout->fieldWidth(); + *fout << map << unicsv_fieldsep + << qSetFieldWidth(5) << qSetRealNumberPrecision(0) << east << qSetFieldWidth(fieldWidth) + << unicsv_fieldsep + << qSetFieldWidth(5) << north << qSetFieldWidth(fieldWidth); break; } case grid_utm: { @@ -1507,11 +1484,10 @@ unicsv_waypt_disp_cb(const Waypoint* wpt) &east, &north, &zone, &zonec, unicsv_datum_idx)) { unicsv_fatal_outside(wpt); } - gbfprintf(fout, "%02d%s%c%s%.0f%s%.0f", - zone, unicsv_fieldsep, - zonec, unicsv_fieldsep, - east, unicsv_fieldsep, - north); + *fout << QString("%1").arg(zone, 2, 10, QLatin1Char('0')) << unicsv_fieldsep + << zonec << unicsv_fieldsep + << qSetRealNumberPrecision(0) << east << unicsv_fieldsep + << north; break; } case grid_swiss: { @@ -1520,13 +1496,14 @@ unicsv_waypt_disp_cb(const Waypoint* wpt) if (! GPS_Math_WGS84_To_Swiss_EN(wpt->latitude, wpt->longitude, &east, &north)) { unicsv_fatal_outside(wpt); } - gbfprintf(fout, "%.f%s%.f", - east, unicsv_fieldsep, north); + *fout << qSetRealNumberPrecision(0) << east << unicsv_fieldsep + << north; break; } default: - gbfprintf(fout, "%.*f%s%.*f", llprec, lat, unicsv_fieldsep, llprec, lon); + *fout << qSetRealNumberPrecision(llprec) << lat << unicsv_fieldsep + << lon; break; } @@ -1539,9 +1516,10 @@ unicsv_waypt_disp_cb(const Waypoint* wpt) } if FIELD_USED(fld_altitude) { if (wpt->altitude != unknown_alt) { - gbfprintf(fout, "%s%.1f", unicsv_fieldsep, wpt->altitude); + *fout << unicsv_fieldsep + << qSetRealNumberPrecision(1) << wpt->altitude; } else { - gbfputs(unicsv_fieldsep, fout); + *fout << unicsv_fieldsep; } } if FIELD_USED(fld_description) { @@ -1555,37 +1533,42 @@ unicsv_waypt_disp_cb(const Waypoint* wpt) } if FIELD_USED(fld_depth) { if WAYPT_HAS(wpt, depth) { - gbfprintf(fout, "%s%.3f", unicsv_fieldsep, wpt->depth); + *fout << unicsv_fieldsep + << qSetRealNumberPrecision(3) << wpt->depth; } else { - gbfputs(unicsv_fieldsep, fout); + *fout << unicsv_fieldsep; } } if FIELD_USED(fld_proximity) { if WAYPT_HAS(wpt, proximity) { - gbfprintf(fout, "%s%.f", unicsv_fieldsep, wpt->proximity); + *fout << unicsv_fieldsep + << qSetRealNumberPrecision(0) << wpt->proximity; } else { - gbfputs(unicsv_fieldsep, fout); + *fout << unicsv_fieldsep; } } if FIELD_USED(fld_temperature) { if WAYPT_HAS(wpt, temperature) { - gbfprintf(fout, "%s%.3f", unicsv_fieldsep, wpt->temperature); + *fout << unicsv_fieldsep + << qSetRealNumberPrecision(3) << wpt->temperature; } else { - gbfputs(unicsv_fieldsep, fout); + *fout << unicsv_fieldsep; } } if FIELD_USED(fld_speed) { if WAYPT_HAS(wpt, speed) { - gbfprintf(fout, "%s%.2f", unicsv_fieldsep, wpt->speed); + *fout << unicsv_fieldsep + << qSetRealNumberPrecision(2) << wpt->speed; } else { - gbfputs(unicsv_fieldsep, fout); + *fout << unicsv_fieldsep; } } if FIELD_USED(fld_course) { if WAYPT_HAS(wpt, course) { - gbfprintf(fout, "%s%.1f", unicsv_fieldsep, wpt->course); + *fout << unicsv_fieldsep + << qSetRealNumberPrecision(1) << wpt->course; } else { - gbfputs(unicsv_fieldsep, fout); + *fout << unicsv_fieldsep; } } if FIELD_USED(fld_fix) { @@ -1612,56 +1595,60 @@ unicsv_waypt_disp_cb(const Waypoint* wpt) if (fix) { unicsv_print_str(fix); } else { - gbfputs(unicsv_fieldsep, fout); + *fout << unicsv_fieldsep; } } if FIELD_USED(fld_hdop) { if (wpt->hdop > 0) { - gbfprintf(fout, "%s%.2f", unicsv_fieldsep, wpt->hdop); + *fout << unicsv_fieldsep + << qSetRealNumberPrecision(2) << wpt->hdop; } else { - gbfputs(unicsv_fieldsep, fout); + *fout << unicsv_fieldsep; } } if FIELD_USED(fld_vdop) { if (wpt->vdop > 0) { - gbfprintf(fout, "%s%.2f", unicsv_fieldsep, wpt->vdop); + *fout << unicsv_fieldsep + << qSetRealNumberPrecision(2) << wpt->vdop; } else { - gbfputs(unicsv_fieldsep, fout); + *fout << unicsv_fieldsep; } } if FIELD_USED(fld_pdop) { if (wpt->pdop > 0) { - gbfprintf(fout, "%s%.2f", unicsv_fieldsep, wpt->pdop); + *fout << unicsv_fieldsep + << qSetRealNumberPrecision(2) << wpt->pdop; } else { - gbfputs(unicsv_fieldsep, fout); + *fout << unicsv_fieldsep; } } if FIELD_USED(fld_sat) { if (wpt->sat > 0) { - gbfprintf(fout, "%s%d", unicsv_fieldsep, wpt->sat); + *fout << unicsv_fieldsep << wpt->sat; } else { - gbfputs(unicsv_fieldsep, fout); + *fout << unicsv_fieldsep; } } if FIELD_USED(fld_heartrate) { if (wpt->heartrate != 0) { - gbfprintf(fout, "%s%u", unicsv_fieldsep, wpt->heartrate); + *fout << unicsv_fieldsep << wpt->heartrate; } else { - gbfputs(unicsv_fieldsep, fout); + *fout << unicsv_fieldsep; } } if FIELD_USED(fld_cadence) { if (wpt->cadence != 0) { - gbfprintf(fout, "%s%u", unicsv_fieldsep, wpt->cadence); + *fout << unicsv_fieldsep << wpt->cadence; } else { - gbfputs(unicsv_fieldsep, fout); + *fout << unicsv_fieldsep; } } if FIELD_USED(fld_power) { if (wpt->power > 0) { - gbfprintf(fout, "%s%.1f", unicsv_fieldsep, wpt->power); + *fout << unicsv_fieldsep + << qSetRealNumberPrecision(1) << wpt->power; } else { - gbfputs(unicsv_fieldsep, fout); + *fout << unicsv_fieldsep; } } if FIELD_USED(fld_date) { @@ -1675,10 +1662,9 @@ unicsv_waypt_disp_cb(const Waypoint* wpt) dt = wpt->GetCreationTime(); } QString date = dt.toString("yyyy/MM/dd"); - gbfputs(unicsv_fieldsep, fout); - gbfputs(date, fout); + *fout << unicsv_fieldsep << date; } else { - gbfputs(unicsv_fieldsep, fout); + *fout << unicsv_fieldsep; } } if FIELD_USED(fld_time) { @@ -1696,10 +1682,9 @@ unicsv_waypt_disp_cb(const Waypoint* wpt) } else { out = t.toString("hh:mm:ss"); } - gbfputs(unicsv_fieldsep, fout); - gbfputs(out, fout); + *fout << unicsv_fieldsep << out; } else { - gbfputs(unicsv_fieldsep, fout); + *fout << unicsv_fieldsep; } } if (FIELD_USED(fld_url)) { @@ -1749,83 +1734,83 @@ unicsv_waypt_disp_cb(const Waypoint* wpt) } if FIELD_USED(fld_gc_id) { - gbfputs(unicsv_fieldsep, fout); + *fout << unicsv_fieldsep; if (gc_data && gc_data->id) { - gbfprintf(fout, "%d", gc_data->id); + *fout << gc_data->id; } } if FIELD_USED(fld_gc_type) { if (gc_data) { unicsv_print_str(gs_get_cachetype(gc_data->type)); } else { - gbfputs(unicsv_fieldsep, fout); + *fout << unicsv_fieldsep; } } if FIELD_USED(fld_gc_container) { if (gc_data) { unicsv_print_str(gs_get_container(gc_data->container)); } else { - gbfputs(unicsv_fieldsep, fout); + *fout << unicsv_fieldsep; } } if FIELD_USED(fld_gc_terr) { - gbfputs(unicsv_fieldsep, fout); + *fout << unicsv_fieldsep; if (gc_data && gc_data->terr) { - gbfprintf(fout, "%.1f", (double)gc_data->terr / 10); + *fout << qSetRealNumberPrecision(1) << ((double)gc_data->terr / 10); } } if FIELD_USED(fld_gc_diff) { - gbfputs(unicsv_fieldsep, fout); + *fout << unicsv_fieldsep; if (gc_data && gc_data->diff) { - gbfprintf(fout, "%.1f", (double)gc_data->diff / 10); + *fout << qSetRealNumberPrecision(1) << ((double)gc_data->diff / 10); } } if FIELD_USED(fld_gc_is_archived) { if (gc_data && gc_data->is_archived) { unicsv_print_str((gc_data->is_archived == status_true) ? "True" : "False"); } else { - gbfputs(unicsv_fieldsep, fout); + *fout << unicsv_fieldsep; } } if FIELD_USED(fld_gc_is_available) { if (gc_data && gc_data->is_available) { unicsv_print_str((gc_data->is_available == status_true) ? "True" : "False"); } else { - gbfputs(unicsv_fieldsep, fout); + *fout << unicsv_fieldsep; } } if FIELD_USED(fld_gc_exported) { if (gc_data) { unicsv_print_data_time(gc_data->exported); } else { - gbfputs(unicsv_fieldsep, fout); + *fout << unicsv_fieldsep; } } if FIELD_USED(fld_gc_last_found) { if (gc_data) { unicsv_print_data_time(gc_data->last_found); } else { - gbfputs(unicsv_fieldsep, fout); + *fout << unicsv_fieldsep; } } if FIELD_USED(fld_gc_placer) { if (gc_data) { unicsv_print_str(gc_data->placer); } else { - gbfputs(unicsv_fieldsep, fout); + *fout << unicsv_fieldsep; } } if FIELD_USED(fld_gc_placer_id) { - gbfputs(unicsv_fieldsep, fout); + *fout << unicsv_fieldsep; if (gc_data && gc_data->placer_id) { - gbfprintf(fout, "%d", gc_data->placer_id); + *fout << gc_data->placer_id; } } if FIELD_USED(fld_gc_hint) { if (gc_data) { unicsv_print_str(gc_data->hint); } else { - gbfputs(unicsv_fieldsep, fout); + *fout << unicsv_fieldsep; } } if (opt_format) { @@ -1835,7 +1820,7 @@ unicsv_waypt_disp_cb(const Waypoint* wpt) unicsv_print_str(wpt->session->filename); } - gbfputs(UNICSV_LINE_SEP, fout); + *fout << UNICSV_LINE_SEP; } /* --------------------------------------------------------------------------- */ @@ -1844,7 +1829,9 @@ unicsv_waypt_disp_cb(const Waypoint* wpt) static void unicsv_wr_init(const QString& filename) { - fout = gbfopen(filename, "wb", MYNAME); + fout = new gpsbabel::TextStream; + fout->open(filename, QIODevice::WriteOnly, MYNAME, opt_codec); + fout->setRealNumberNotation(QTextStream::FixedNotation); memset(&unicsv_outp_flags, 0, sizeof(unicsv_outp_flags)); unicsv_grid_idx = grid_unknown; @@ -1884,7 +1871,9 @@ unicsv_wr_init(const QString& filename) static void unicsv_wr_deinit() { - gbfclose(fout); + fout->close(); + delete fout; + fout = nullptr; } // Waypoints are default-on and there's no way to turn them off. This is @@ -1923,170 +1912,168 @@ unicsv_wr() Fatal() << MYNAME << ": Realtime positioning not supported."; } - gbfprintf(fout, "No%s", unicsv_fieldsep); + *fout << "No" << unicsv_fieldsep; switch (unicsv_grid_idx) { case grid_bng: - /* indexed parameters doesn't work under __win32__ (mingw) - gbfprintf(fout, "BNG-Zone%1$sBNG-East%1$sBNG-North", unicsv_fieldsep); - */ - gbfprintf(fout, "BNG-Zone%sBNG-East%sBNG-North", - unicsv_fieldsep, unicsv_fieldsep); + *fout << "BNG-Zone" << unicsv_fieldsep + << "BNG-East" << unicsv_fieldsep + << "BNG-North"; break; case grid_utm: - /* indexed parameters doesn't work under __win32__ (mingw) - gbfprintf(fout, "BNG-Zone%1$sBNG-East%1$sBNG-North", unicsv_fieldsep); - */ - gbfprintf(fout, "UTM-Zone%sUTM-Ch%sUTM-East%sUTM-North", - unicsv_fieldsep, unicsv_fieldsep, unicsv_fieldsep); + *fout << "UTM-Zone" << unicsv_fieldsep + << "UTM-Ch" << unicsv_fieldsep + << "UTM-East" << unicsv_fieldsep + << "UTM-North"; break; case grid_swiss: - gbfprintf(fout, "Swiss-East%sSwiss-North", - unicsv_fieldsep); + *fout << "Swiss-East" << unicsv_fieldsep + << "Swiss-North"; break; default: - gbfprintf(fout, "Latitude%sLongitude", unicsv_fieldsep); + *fout << "Latitude" << unicsv_fieldsep + << "Longitude"; } if FIELD_USED(fld_shortname) { - gbfprintf(fout, "%sName", unicsv_fieldsep); + *fout << unicsv_fieldsep << "Name"; } if FIELD_USED(fld_altitude) { - gbfprintf(fout, "%sAltitude", unicsv_fieldsep); + *fout << unicsv_fieldsep << "Altitude"; } if FIELD_USED(fld_description) { - gbfprintf(fout, "%sDescription", unicsv_fieldsep); + *fout << unicsv_fieldsep << "Description"; } if FIELD_USED(fld_notes) { - gbfprintf(fout, "%sNotes", unicsv_fieldsep); + *fout << unicsv_fieldsep << "Notes"; } if FIELD_USED(fld_symbol) { - gbfprintf(fout, "%sSymbol", unicsv_fieldsep); + *fout << unicsv_fieldsep << "Symbol"; } if FIELD_USED(fld_depth) { - gbfprintf(fout, "%sDepth", unicsv_fieldsep); + *fout << unicsv_fieldsep << "Depth"; } if FIELD_USED(fld_proximity) { - gbfprintf(fout, "%sProximity", unicsv_fieldsep); + *fout << unicsv_fieldsep << "Proximity"; } if FIELD_USED(fld_temperature) { - gbfprintf(fout, "%sTemperature", unicsv_fieldsep); + *fout << unicsv_fieldsep << "Temperature"; } if FIELD_USED(fld_speed) { - gbfprintf(fout, "%sSpeed", unicsv_fieldsep); + *fout << unicsv_fieldsep << "Speed"; } if FIELD_USED(fld_course) { - gbfprintf(fout, "%sCourse", unicsv_fieldsep); + *fout << unicsv_fieldsep << "Course"; } if FIELD_USED(fld_fix) { - gbfprintf(fout, "%sFIX", unicsv_fieldsep); + *fout << unicsv_fieldsep << "FIX"; } if FIELD_USED(fld_hdop) { - gbfprintf(fout, "%sHDOP", unicsv_fieldsep); + *fout << unicsv_fieldsep << "HDOP"; } if FIELD_USED(fld_vdop) { - gbfprintf(fout, "%sVDOP", unicsv_fieldsep); + *fout << unicsv_fieldsep << "VDOP"; } if FIELD_USED(fld_pdop) { - gbfprintf(fout, "%sPDOP", unicsv_fieldsep); + *fout << unicsv_fieldsep << "PDOP"; } if FIELD_USED(fld_sat) { - gbfprintf(fout, "%sSatellites", unicsv_fieldsep); + *fout << unicsv_fieldsep << "Satellites"; } if FIELD_USED(fld_heartrate) { - gbfprintf(fout, "%sHeartrate", unicsv_fieldsep); + *fout << unicsv_fieldsep << "Heartrate"; } if FIELD_USED(fld_cadence) { - gbfprintf(fout, "%sCadence", unicsv_fieldsep); + *fout << unicsv_fieldsep << "Cadence"; } if FIELD_USED(fld_power) { - gbfprintf(fout, "%sPower", unicsv_fieldsep); + *fout << unicsv_fieldsep << "Power"; } if FIELD_USED(fld_date) { - gbfprintf(fout, "%sDate", unicsv_fieldsep); + *fout << unicsv_fieldsep << "Date"; } if FIELD_USED(fld_time) { - gbfprintf(fout, "%sTime", unicsv_fieldsep); + *fout << unicsv_fieldsep << "Time"; } if FIELD_USED(fld_url) { - gbfprintf(fout, "%sURL", unicsv_fieldsep); + *fout << unicsv_fieldsep << "URL"; } if FIELD_USED(fld_garmin_facility) { - gbfprintf(fout, "%sFacility", unicsv_fieldsep); + *fout << unicsv_fieldsep << "Facility"; } if FIELD_USED(fld_garmin_addr) { - gbfprintf(fout, "%sAddress", unicsv_fieldsep); + *fout << unicsv_fieldsep << "Address"; } if FIELD_USED(fld_garmin_city) { - gbfprintf(fout, "%sCity", unicsv_fieldsep); + *fout << unicsv_fieldsep << "City"; } if FIELD_USED(fld_garmin_postal_code) { - gbfprintf(fout, "%sPostalCode", unicsv_fieldsep); + *fout << unicsv_fieldsep << "PostalCode"; } if FIELD_USED(fld_garmin_state) { - gbfprintf(fout, "%sState", unicsv_fieldsep); + *fout << unicsv_fieldsep << "State"; } if FIELD_USED(fld_garmin_country) { - gbfprintf(fout, "%sCountry", unicsv_fieldsep); + *fout << unicsv_fieldsep << "Country"; } if FIELD_USED(fld_garmin_phone_nr) { - gbfprintf(fout, "%sPhone", unicsv_fieldsep); + *fout << unicsv_fieldsep << "Phone"; } if FIELD_USED(fld_garmin_phone_nr2) { - gbfprintf(fout, "%sPhone2", unicsv_fieldsep); + *fout << unicsv_fieldsep << "Phone2"; } if FIELD_USED(fld_garmin_fax_nr) { - gbfprintf(fout, "%sFax", unicsv_fieldsep); + *fout << unicsv_fieldsep << "Fax"; } if FIELD_USED(fld_garmin_email) { - gbfprintf(fout, "%sEmail", unicsv_fieldsep); + *fout << unicsv_fieldsep << "Email"; } if FIELD_USED(fld_gc_id) { - gbfprintf(fout, "%sGCID", unicsv_fieldsep); + *fout << unicsv_fieldsep << "GCID"; } if FIELD_USED(fld_gc_type) { - gbfprintf(fout, "%sType", unicsv_fieldsep); + *fout << unicsv_fieldsep << "Type"; } if FIELD_USED(fld_gc_container) { - gbfprintf(fout, "%sContainer", unicsv_fieldsep); + *fout << unicsv_fieldsep << "Container"; } if FIELD_USED(fld_gc_terr) { - gbfprintf(fout, "%sTerrain", unicsv_fieldsep); + *fout << unicsv_fieldsep << "Terrain"; } if FIELD_USED(fld_gc_diff) { - gbfprintf(fout, "%sDifficulty", unicsv_fieldsep); + *fout << unicsv_fieldsep << "Difficulty"; } if FIELD_USED(fld_gc_is_archived) { - gbfprintf(fout, "%sArchived", unicsv_fieldsep); + *fout << unicsv_fieldsep << "Archived"; } if FIELD_USED(fld_gc_is_available) { - gbfprintf(fout, "%sAvailable", unicsv_fieldsep); + *fout << unicsv_fieldsep << "Available"; } if FIELD_USED(fld_gc_exported) { - gbfprintf(fout, "%sExported", unicsv_fieldsep); + *fout << unicsv_fieldsep << "Exported"; } if FIELD_USED(fld_gc_last_found) { - gbfprintf(fout, "%sLast Found", unicsv_fieldsep); + *fout << unicsv_fieldsep << "Last Found"; } if FIELD_USED(fld_gc_placer) { - gbfprintf(fout, "%sPlacer", unicsv_fieldsep); + *fout << unicsv_fieldsep << "Placer"; } if FIELD_USED(fld_gc_placer_id) { - gbfprintf(fout, "%sPlacer ID", unicsv_fieldsep); + *fout << unicsv_fieldsep << "Placer ID"; } if FIELD_USED(fld_gc_hint) { - gbfprintf(fout, "%sHint", unicsv_fieldsep); + *fout << unicsv_fieldsep << "Hint"; } if (opt_format) { - gbfprintf(fout, "%sFormat", unicsv_fieldsep); + *fout << unicsv_fieldsep << "Format"; } if (opt_filename) { - gbfprintf(fout, "%sFilename", unicsv_fieldsep); + *fout << unicsv_fieldsep << "Filename"; } - gbfputs(UNICSV_LINE_SEP, fout); + *fout << UNICSV_LINE_SEP; switch (global_opts.objective) { case wptdata: @@ -2116,7 +2103,7 @@ ff_vecs_t unicsv_vecs = { unicsv_wr, nullptr, unicsv_args, - CET_CHARSET_ASCII, 0 /* can be changed with -c ... */ + CET_CHARSET_UTF8, 0 , NULL_POS_OPS, nullptr }; diff --git a/util.cc b/util.cc index 78986e434..5f988f421 100644 --- a/util.cc +++ b/util.cc @@ -19,19 +19,33 @@ */ +#include // for isspace, isalpha, ispunct, tolower, toupper +#include // for errno +#include // for fabs, floor +#include // for va_list, va_end, va_start, va_copy +#include // for uint32_t +#include // for size_t, vsnprintf, FILE, fopen, printf, sprintf, stderr, stdin, stdout +#include // for abs, getenv, calloc, free, malloc, realloc +#include // for strlen, strcat, strstr, memcpy, strcmp, strcpy, strdup, strchr, strerror +#include // for mktime, localtime + +#include // for QByteArray +#include // for QChar, operator<=, operator>= +#include // for QCharRef +#include // for QDateTime +#include // for QFileInfo +#include // for QList +#include // for QString, operator+ +#include // for QTextCodec +#include // for operator<<, QTextStream, qSetFieldWidth, endl, QTextStream::AlignLeft +#include // for CaseInsensitive +#include // for qPrintable + #include "defs.h" -#include "jeeps/gpsmath.h" +#include "cet.h" // for cet_utf8_to_ucs4 +#include "src/core/datetime.h" // for DateTime #include "src/core/xmltag.h" -#include -#include -#include -#include -#include -#include // for va_copy -#include -#include -#include // First test Apple's clever macro that's really a runtime test so // that our universal binaries work right. @@ -479,22 +493,6 @@ str_match(const char* str, const char* match) return ((*s == '\0') && (*m == '\0')); } -// for ruote_char = " -// make str = blank into nothing -// make str = foo into "foo" -// make str = foo"bar into "foo""bar" -// No, that doesn't seem obvious to me, either... - -QString -strenquote(const QString& str, const QChar quot_char) -{ - QString replacement = QString("%1%1").arg(quot_char); - QString t = str; - t.replace(quot_char, replacement); - QString r = quot_char + t + quot_char; - return r; -} - void printposn(const double c, int is_lat) { @@ -1730,3 +1728,36 @@ int gb_ptr2int(const void* p) return x.i; } + +void +list_codecs() +{ + QTextStream info(stderr); + info.setFieldAlignment(QTextStream::AlignLeft); + auto mibs = QTextCodec::availableMibs(); + int maxlen = 0; + for (auto mib : mibs) { + auto codec = QTextCodec::codecForMib(mib); + if (codec->name().size() > maxlen) { + maxlen = codec->name().size(); + } + } + info << "Avaialble Codecs:" << endl; + info << qSetFieldWidth(8) << "MIBenum" << qSetFieldWidth(maxlen+1) << "Name" << qSetFieldWidth(0) << "Aliases" << endl; + for (auto mib : mibs) { + auto codec = QTextCodec::codecForMib(mib); + info << qSetFieldWidth(8) << mib << qSetFieldWidth(maxlen+1) << codec->name() << qSetFieldWidth(0); + bool first = true; + const auto aliases = codec->aliases(); + for (const auto& alias : aliases) { + if (first) { + first = false; + } else { + info << ", "; + } + info << alias; + } + info << endl; + } +} + diff --git a/xcsv.cc b/xcsv.cc index eee9399f5..1383be5d9 100644 --- a/xcsv.cc +++ b/xcsv.cc @@ -49,7 +49,7 @@ #include // for qAsConst, QAddConst<>::Type, qPrintable #include "defs.h" -#include "csv_util.h" // for csv_stringtrim, dec_to_human, csv_stringclean, csv_lineparse, human_to_dec, ddmmdir_to_degrees, dec_to_intdeg, decdir_to_dec, intdeg_to_dec +#include "csv_util.h" // for csv_stringtrim, dec_to_human, csv_stringclean, human_to_dec, ddmmdir_to_degrees, dec_to_intdeg, decdir_to_dec, intdeg_to_dec, csv_linesplit #include "garmin_fs.h" // for garmin_fs_t, garmin_fs_flags_t, GMSD_FIND, GMSD_GET, GMSD_SET, garmin_fs_alloc #include "gbfile.h" // for gbfgetstr, gbfclose, gbfopen, gbfile #include "grtcirc.h" // for RAD, gcdist, radtomiles @@ -1056,11 +1056,8 @@ xcsv_data_read() Waypoint* wpt_tmp = new Waypoint; // initialize parse data for accumulation of line results from all fields in this line. xcsv_parse_data parse_data; - // tbuf is a temporary copy of buff since we modify it. :-( - char *tbuf = xstrdup(buff); - const char* s = tbuf; - s = csv_lineparse(s, CSTR(xcsv_file.field_delimiter), - CSTR(xcsv_file.field_encloser), linecount); + const QStringList values = csv_linesplit(buff, xcsv_file.field_delimiter, + xcsv_file.field_encloser, linecount); if (xcsv_file.ifields.isEmpty()) { fatal(MYNAME ": attempt to read, but style '%s' has no IFIELDs in it.\n", CSTR(xcsv_file.description)? CSTR(xcsv_file.description) : "unknown"); @@ -1068,23 +1065,15 @@ xcsv_data_read() int ifield_idx = 0; - /* now rip the line apart, advancing the queue for each tear - * off the beginning of buff since there's no index into queue. - */ - while (s) { + /* now rip the line apart */ + for (const auto& value : values) { const field_map& fmp = xcsv_file.ifields.at(ifield_idx++); - xcsv_parse_val(s, wpt_tmp, fmp, &parse_data, linecount); + xcsv_parse_val(CSTR(value), wpt_tmp, fmp, &parse_data, linecount); if (ifield_idx >= xcsv_file.ifields.size()) { - /* we've wrapped the queue. so stop parsing! */ - while (s) { - s = csv_lineparse(nullptr, "\xff","",linecount); - } + /* no more fields, stop parsing! */ break; } - - s = csv_lineparse(nullptr, CSTR(xcsv_file.field_delimiter), - CSTR(xcsv_file.field_encloser), linecount); } // If XT_LAT_DIR(XT_LON_DIR) was an input field, and the latitude(longitude) is positive, @@ -1144,7 +1133,6 @@ xcsv_data_read() default: ; } - xfree(tbuf); } } } diff --git a/xmldoc/formats/options/unicsv-codec.xml b/xmldoc/formats/options/unicsv-codec.xml new file mode 100644 index 000000000..98eca60b4 --- /dev/null +++ b/xmldoc/formats/options/unicsv-codec.xml @@ -0,0 +1,5 @@ + +This lets you override the default codec of 'UTF-8'. As an +input option the codec should correspond to the encoding of the input file. +As an output option it sets the encoding of the output file. +